1 package org.apache.lucene.queryparser.util;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 import java.io.IOException;
21 import java.text.DateFormat;
22 import java.util.Calendar;
23 import java.util.Date;
24 import java.util.GregorianCalendar;
25 import java.util.Locale;
26 import java.util.TimeZone;
27
28 import org.apache.lucene.analysis.*;
29 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
30 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
31 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
32 import org.apache.lucene.document.DateTools;
33 import org.apache.lucene.document.Document;
34 import org.apache.lucene.document.Field;
35 import org.apache.lucene.index.DirectoryReader;
36 import org.apache.lucene.index.IndexReader;
37 import org.apache.lucene.index.IndexWriter;
38 import org.apache.lucene.index.Term;
39
40
41
42
43 import org.apache.lucene.queryparser.classic.QueryParserBase;
44
45 import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration;
46 import org.apache.lucene.search.*;
47 import org.apache.lucene.search.BooleanClause.Occur;
48 import org.apache.lucene.store.Directory;
49 import org.apache.lucene.util.LuceneTestCase;
50 import org.apache.lucene.util.automaton.Automata;
51 import org.apache.lucene.util.automaton.CharacterRunAutomaton;
52 import org.apache.lucene.util.automaton.RegExp;
53 import org.junit.AfterClass;
54 import org.junit.BeforeClass;
55
56
57
58
59
60
61 public abstract class QueryParserTestBase extends LuceneTestCase {
62
63 public static Analyzer qpAnalyzer;
64
65 @BeforeClass
66 public static void beforeClass() {
67 qpAnalyzer = new QPTestAnalyzer();
68 }
69
70 @AfterClass
71 public static void afterClass() {
72 qpAnalyzer = null;
73 }
74
75 public static final class QPTestFilter extends TokenFilter {
76 CharTermAttribute termAtt;
77 OffsetAttribute offsetAtt;
78
79
80
81
82
83 public QPTestFilter(TokenStream in) {
84 super(in);
85 termAtt = addAttribute(CharTermAttribute.class);
86 offsetAtt = addAttribute(OffsetAttribute.class);
87 }
88
89 boolean inPhrase = false;
90 int savedStart = 0, savedEnd = 0;
91
92 @Override
93 public boolean incrementToken() throws IOException {
94 if (inPhrase) {
95 inPhrase = false;
96 clearAttributes();
97 termAtt.append("phrase2");
98 offsetAtt.setOffset(savedStart, savedEnd);
99 return true;
100 } else
101 while (input.incrementToken()) {
102 if (termAtt.toString().equals("phrase")) {
103 inPhrase = true;
104 savedStart = offsetAtt.startOffset();
105 savedEnd = offsetAtt.endOffset();
106 termAtt.setEmpty().append("phrase1");
107 offsetAtt.setOffset(savedStart, savedEnd);
108 return true;
109 } else if (!termAtt.toString().equals("stop"))
110 return true;
111 }
112 return false;
113 }
114 }
115
116 public static final class QPTestAnalyzer extends Analyzer {
117
118
119 @Override
120 public TokenStreamComponents createComponents(String fieldName) {
121 Tokenizer tokenizer = new MockTokenizer(MockTokenizer.SIMPLE, true);
122 return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer));
123 }
124 }
125
126 private int originalMaxClauses;
127
128 private String defaultField = "field";
129
130 protected String getDefaultField(){
131 return defaultField;
132 }
133
134 protected void setDefaultField(String defaultField){
135 this.defaultField = defaultField;
136 }
137
138 @Override
139 public void setUp() throws Exception {
140 super.setUp();
141 originalMaxClauses = BooleanQuery.getMaxClauseCount();
142 }
143
144 public abstract CommonQueryParserConfiguration getParserConfig(Analyzer a) throws Exception;
145
146 public abstract void setDefaultOperatorOR(CommonQueryParserConfiguration cqpC);
147
148 public abstract void setDefaultOperatorAND(CommonQueryParserConfiguration cqpC);
149
150 public abstract void setAnalyzeRangeTerms(CommonQueryParserConfiguration cqpC, boolean value);
151
152 public abstract void setAutoGeneratePhraseQueries(CommonQueryParserConfiguration cqpC, boolean value);
153
154 public abstract void setDateResolution(CommonQueryParserConfiguration cqpC, CharSequence field, DateTools.Resolution value);
155
156 public abstract Query getQuery(String query, CommonQueryParserConfiguration cqpC) throws Exception;
157
158 public abstract Query getQuery(String query, Analyzer a) throws Exception;
159
160 public abstract boolean isQueryParserException(Exception exception);
161
162 public Query getQuery(String query) throws Exception {
163 return getQuery(query, (Analyzer)null);
164 }
165
166 public void assertQueryEquals(String query, Analyzer a, String result)
167 throws Exception {
168 Query q = getQuery(query, a);
169 String s = q.toString("field");
170 if (!s.equals(result)) {
171 fail("Query /" + query + "/ yielded /" + s
172 + "/, expecting /" + result + "/");
173 }
174 }
175
176 public void assertQueryEquals(CommonQueryParserConfiguration cqpC, String field, String query, String result)
177 throws Exception {
178 Query q = getQuery(query, cqpC);
179 String s = q.toString(field);
180 if (!s.equals(result)) {
181 fail("Query /" + query + "/ yielded /" + s
182 + "/, expecting /" + result + "/");
183 }
184 }
185
186 public void assertEscapedQueryEquals(String query, Analyzer a, String result)
187 throws Exception {
188 String escapedQuery = QueryParserBase.escape(query);
189 if (!escapedQuery.equals(result)) {
190 fail("Query /" + query + "/ yielded /" + escapedQuery
191 + "/, expecting /" + result + "/");
192 }
193 }
194
195 public void assertWildcardQueryEquals(String query, boolean lowercase, String result, boolean allowLeadingWildcard)
196 throws Exception {
197 CommonQueryParserConfiguration cqpC = getParserConfig(null);
198 cqpC.setLowercaseExpandedTerms(lowercase);
199 cqpC.setAllowLeadingWildcard(allowLeadingWildcard);
200 Query q = getQuery(query, cqpC);
201 String s = q.toString("field");
202 if (!s.equals(result)) {
203 fail("WildcardQuery /" + query + "/ yielded /" + s
204 + "/, expecting /" + result + "/");
205 }
206 }
207
208 public void assertWildcardQueryEquals(String query, boolean lowercase, String result)
209 throws Exception {
210 assertWildcardQueryEquals(query, lowercase, result, false);
211 }
212
213 public void assertWildcardQueryEquals(String query, String result) throws Exception {
214 Query q = getQuery(query);
215 String s = q.toString("field");
216 if (!s.equals(result)) {
217 fail("WildcardQuery /" + query + "/ yielded /" + s + "/, expecting /"
218 + result + "/");
219 }
220 }
221
222 public Query getQueryDOA(String query, Analyzer a)
223 throws Exception {
224 if (a == null)
225 a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
226 CommonQueryParserConfiguration qp = getParserConfig(a);
227 setDefaultOperatorAND(qp);
228 return getQuery(query, qp);
229 }
230
231 public void assertQueryEqualsDOA(String query, Analyzer a, String result)
232 throws Exception {
233 Query q = getQueryDOA(query, a);
234 String s = q.toString("field");
235 if (!s.equals(result)) {
236 fail("Query /" + query + "/ yielded /" + s
237 + "/, expecting /" + result + "/");
238 }
239 }
240
241 public void testCJK() throws Exception {
242
243
244 assertQueryEquals("term\u3000term\u3000term", null, "term\u0020term\u0020term");
245 assertQueryEquals("用語\u3000用語\u3000用語", null, "用語\u0020用語\u0020用語");
246 }
247
248
249 protected static class SimpleCJKTokenizer extends Tokenizer {
250 private CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
251
252 public SimpleCJKTokenizer() {
253 super();
254 }
255
256 @Override
257 public final boolean incrementToken() throws IOException {
258 int ch = input.read();
259 if (ch < 0)
260 return false;
261 clearAttributes();
262 termAtt.setEmpty().append((char) ch);
263 return true;
264 }
265 }
266
267 private class SimpleCJKAnalyzer extends Analyzer {
268 @Override
269 public TokenStreamComponents createComponents(String fieldName) {
270 return new TokenStreamComponents(new SimpleCJKTokenizer());
271 }
272 }
273
274 public void testCJKTerm() throws Exception {
275
276 SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
277
278 BooleanQuery.Builder expected = new BooleanQuery.Builder();
279 expected.add(new TermQuery(new Term("field", "中")), BooleanClause.Occur.SHOULD);
280 expected.add(new TermQuery(new Term("field", "国")), BooleanClause.Occur.SHOULD);
281
282 assertEquals(expected.build(), getQuery("中国", analyzer));
283 }
284
285 public void testCJKBoostedTerm() throws Exception {
286
287 SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
288
289 BooleanQuery.Builder expectedB = new BooleanQuery.Builder();
290 expectedB.add(new TermQuery(new Term("field", "中")), BooleanClause.Occur.SHOULD);
291 expectedB.add(new TermQuery(new Term("field", "国")), BooleanClause.Occur.SHOULD);
292 Query expected = expectedB.build();
293 expected = new BoostQuery(expected, 0.5f);
294
295 assertEquals(expected, getQuery("中国^0.5", analyzer));
296 }
297
298 public void testCJKPhrase() throws Exception {
299
300 SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
301
302 PhraseQuery expected = new PhraseQuery("field", "中", "国");
303
304 assertEquals(expected, getQuery("\"中国\"", analyzer));
305 }
306
307 public void testCJKBoostedPhrase() throws Exception {
308
309 SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
310
311 Query expected = new PhraseQuery("field", "中", "国");
312 expected = new BoostQuery(expected, 0.5f);
313
314 assertEquals(expected, getQuery("\"中国\"^0.5", analyzer));
315 }
316
317 public void testCJKSloppyPhrase() throws Exception {
318
319 SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
320
321 PhraseQuery expected = new PhraseQuery(3, "field", "中", "国");
322
323 assertEquals(expected, getQuery("\"中国\"~3", analyzer));
324 }
325
326 public void testAutoGeneratePhraseQueriesOn() throws Exception {
327
328 SimpleCJKAnalyzer analyzer = new SimpleCJKAnalyzer();
329
330 PhraseQuery expected = new PhraseQuery("field", "中", "国");
331 CommonQueryParserConfiguration qp = getParserConfig(analyzer);
332 setAutoGeneratePhraseQueries(qp, true);
333 assertEquals(expected, getQuery("中国",qp));
334 }
335
336 public void testSimple() throws Exception {
337 assertQueryEquals("term term term", null, "term term term");
338 assertQueryEquals("türm term term", new MockAnalyzer(random()), "türm term term");
339 assertQueryEquals("ümlaut", new MockAnalyzer(random()), "ümlaut");
340
341
342
343
344
345
346 assertQueryEquals("a AND b", null, "+a +b");
347 assertQueryEquals("(a AND b)", null, "+a +b");
348 assertQueryEquals("c OR (a AND b)", null, "c (+a +b)");
349 assertQueryEquals("a AND NOT b", null, "+a -b");
350 assertQueryEquals("a AND -b", null, "+a -b");
351 assertQueryEquals("a AND !b", null, "+a -b");
352 assertQueryEquals("a && b", null, "+a +b");
353
354
355 assertQueryEquals("a OR b", null, "a b");
356 assertQueryEquals("a || b", null, "a b");
357 assertQueryEquals("a OR !b", null, "a -b");
358
359 assertQueryEquals("a OR -b", null, "a -b");
360
361 assertQueryEquals("+term -term term", null, "+term -term term");
362 assertQueryEquals("foo:term AND field:anotherTerm", null,
363 "+foo:term +anotherterm");
364 assertQueryEquals("term AND \"phrase phrase\"", null,
365 "+term +\"phrase phrase\"");
366 assertQueryEquals("\"hello there\"", null, "\"hello there\"");
367 assertTrue(getQuery("a AND b") instanceof BooleanQuery);
368 assertTrue(getQuery("hello") instanceof TermQuery);
369 assertTrue(getQuery("\"hello there\"") instanceof PhraseQuery);
370
371 assertQueryEquals("germ term^2.0", null, "germ term^2.0");
372 assertQueryEquals("(term)^2.0", null, "term^2.0");
373 assertQueryEquals("(germ term)^2.0", null, "(germ term)^2.0");
374 assertQueryEquals("term^2.0", null, "term^2.0");
375 assertQueryEquals("term^2", null, "term^2.0");
376 assertQueryEquals("\"germ term\"^2.0", null, "\"germ term\"^2.0");
377 assertQueryEquals("\"term germ\"^2", null, "\"term germ\"^2.0");
378
379 assertQueryEquals("(foo OR bar) AND (baz OR boo)", null,
380 "+(foo bar) +(baz boo)");
381 assertQueryEquals("((a OR b) AND NOT c) OR d", null,
382 "(+(a b) -c) d");
383 assertQueryEquals("+(apple \"steve jobs\") -(foo bar baz)", null,
384 "+(apple \"steve jobs\") -(foo bar baz)");
385 assertQueryEquals("+title:(dog OR cat) -author:\"bob dole\"", null,
386 "+(title:dog title:cat) -author:\"bob dole\"");
387
388 }
389
390 public abstract void testDefaultOperator() throws Exception;
391
392
393 public void testOperatorVsWhitespace() throws Exception {
394
395 Analyzer a = new Analyzer() {
396 @Override
397 public TokenStreamComponents createComponents(String fieldName) {
398 return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, false));
399 }
400 };
401 assertQueryEquals("a - b", a, "a - b");
402 assertQueryEquals("a + b", a, "a + b");
403 assertQueryEquals("a ! b", a, "a ! b");
404 }
405
406 public void testPunct() throws Exception {
407 Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
408 assertQueryEquals("a&b", a, "a&b");
409 assertQueryEquals("a&&b", a, "a&&b");
410 assertQueryEquals(".NET", a, ".NET");
411 }
412
413 public void testSlop() throws Exception {
414 assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
415 assertQueryEquals("\"term germ\"~2 flork", null, "\"term germ\"~2 flork");
416 assertQueryEquals("\"term\"~2", null, "term");
417 assertQueryEquals("\" \"~2 germ", null, "germ");
418 assertQueryEquals("\"term germ\"~2^2", null, "\"term germ\"~2^2.0");
419 }
420
421 public void testNumber() throws Exception {
422
423 assertQueryEquals("3", null, "");
424 assertQueryEquals("term 1.0 1 2", null, "term");
425 assertQueryEquals("term term1 term2", null, "term term term");
426
427 Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true);
428 assertQueryEquals("3", a, "3");
429 assertQueryEquals("term 1.0 1 2", a, "term 1.0 1 2");
430 assertQueryEquals("term term1 term2", a, "term term1 term2");
431 }
432
433 public void testWildcard() throws Exception {
434 assertQueryEquals("term*", null, "term*");
435 assertQueryEquals("term*^2", null, "term*^2.0");
436 assertQueryEquals("term~", null, "term~2");
437 assertQueryEquals("term~1", null, "term~1");
438 assertQueryEquals("term~0.7", null, "term~1");
439 assertQueryEquals("term~^3", null, "term~2^3.0");
440 assertQueryEquals("term^3~", null, "term~2^3.0");
441 assertQueryEquals("term*germ", null, "term*germ");
442 assertQueryEquals("term*germ^3", null, "term*germ^3.0");
443
444 assertTrue(getQuery("term*") instanceof PrefixQuery);
445 assertTrue(getQuery("term*^2") instanceof BoostQuery);
446 assertTrue(((BoostQuery) getQuery("term*^2")).getQuery() instanceof PrefixQuery);
447 assertTrue(getQuery("term~") instanceof FuzzyQuery);
448 assertTrue(getQuery("term~0.7") instanceof FuzzyQuery);
449 FuzzyQuery fq = (FuzzyQuery)getQuery("term~0.7");
450 assertEquals(1, fq.getMaxEdits());
451 assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
452 fq = (FuzzyQuery)getQuery("term~");
453 assertEquals(2, fq.getMaxEdits());
454 assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength());
455
456 assertParseException("term~1.1");
457
458 assertTrue(getQuery("term*germ") instanceof WildcardQuery);
459
460
461
462
463
464
465 assertWildcardQueryEquals("Term*", true, "term*");
466
467 assertWildcardQueryEquals("term*", true, "term*");
468 assertWildcardQueryEquals("Term*", true, "term*");
469 assertWildcardQueryEquals("TERM*", true, "term*");
470
471 assertWildcardQueryEquals("term*", false, "term*");
472 assertWildcardQueryEquals("Term*", false, "Term*");
473 assertWildcardQueryEquals("TERM*", false, "TERM*");
474
475
476 assertWildcardQueryEquals("Te?m", "te?m");
477
478 assertWildcardQueryEquals("te?m", true, "te?m");
479 assertWildcardQueryEquals("Te?m", true, "te?m");
480 assertWildcardQueryEquals("TE?M", true, "te?m");
481 assertWildcardQueryEquals("Te?m*gerM", true, "te?m*germ");
482
483 assertWildcardQueryEquals("te?m", false, "te?m");
484 assertWildcardQueryEquals("Te?m", false, "Te?m");
485 assertWildcardQueryEquals("TE?M", false, "TE?M");
486 assertWildcardQueryEquals("Te?m*gerM", false, "Te?m*gerM");
487
488 assertWildcardQueryEquals("Term~", "term~2");
489 assertWildcardQueryEquals("Term~", true, "term~2");
490 assertWildcardQueryEquals("Term~", false, "Term~2");
491
492 assertWildcardQueryEquals("[A TO C]", "[a TO c]");
493 assertWildcardQueryEquals("[A TO C]", true, "[a TO c]");
494 assertWildcardQueryEquals("[A TO C]", false, "[A TO C]");
495
496 try {
497 assertWildcardQueryEquals("*Term", true, "*term");
498 } catch(Exception pe) {
499
500 if(!isQueryParserException(pe)){
501 fail();
502 }
503 }
504 try {
505 assertWildcardQueryEquals("?Term", true, "?term");
506 fail();
507 } catch(Exception pe) {
508
509 if(!isQueryParserException(pe)){
510 fail();
511 }
512 }
513
514 assertWildcardQueryEquals("*Term", true, "*term", true);
515 assertWildcardQueryEquals("?Term", true, "?term", true);
516 }
517
518 public void testLeadingWildcardType() throws Exception {
519 CommonQueryParserConfiguration cqpC = getParserConfig(null);
520 cqpC.setAllowLeadingWildcard(true);
521 assertEquals(WildcardQuery.class, getQuery("t*erm*",cqpC).getClass());
522 assertEquals(WildcardQuery.class, getQuery("?term*",cqpC).getClass());
523 assertEquals(WildcardQuery.class, getQuery("*term*",cqpC).getClass());
524 }
525
526 public void testQPA() throws Exception {
527 assertQueryEquals("term term^3.0 term", qpAnalyzer, "term term^3.0 term");
528 assertQueryEquals("term stop^3.0 term", qpAnalyzer, "term term");
529
530 assertQueryEquals("term term term", qpAnalyzer, "term term term");
531 assertQueryEquals("term +stop term", qpAnalyzer, "term term");
532 assertQueryEquals("term -stop term", qpAnalyzer, "term term");
533
534 assertQueryEquals("drop AND (stop) AND roll", qpAnalyzer, "+drop +roll");
535 assertQueryEquals("term +(stop) term", qpAnalyzer, "term term");
536 assertQueryEquals("term -(stop) term", qpAnalyzer, "term term");
537
538 assertQueryEquals("drop AND stop AND roll", qpAnalyzer, "+drop +roll");
539 assertQueryEquals("term phrase term", qpAnalyzer,
540 "term (phrase1 phrase2) term");
541 assertQueryEquals("term AND NOT phrase term", qpAnalyzer,
542 "+term -(phrase1 phrase2) term");
543 assertQueryEquals("stop^3", qpAnalyzer, "");
544 assertQueryEquals("stop", qpAnalyzer, "");
545 assertQueryEquals("(stop)^3", qpAnalyzer, "");
546 assertQueryEquals("((stop))^3", qpAnalyzer, "");
547 assertQueryEquals("(stop^3)", qpAnalyzer, "");
548 assertQueryEquals("((stop)^3)", qpAnalyzer, "");
549 assertQueryEquals("(stop)", qpAnalyzer, "");
550 assertQueryEquals("((stop))", qpAnalyzer, "");
551 assertTrue(getQuery("term term term", qpAnalyzer) instanceof BooleanQuery);
552 assertTrue(getQuery("term +stop", qpAnalyzer) instanceof TermQuery);
553
554 CommonQueryParserConfiguration cqpc = getParserConfig(qpAnalyzer);
555 setDefaultOperatorAND(cqpc);
556 assertQueryEquals(cqpc, "field", "term phrase term",
557 "+term +(+phrase1 +phrase2) +term");
558 assertQueryEquals(cqpc, "field", "phrase",
559 "+phrase1 +phrase2");
560 }
561
562 public void testRange() throws Exception {
563 assertQueryEquals("[ a TO z]", null, "[a TO z]");
564 assertQueryEquals("[ a TO z}", null, "[a TO z}");
565 assertQueryEquals("{ a TO z]", null, "{a TO z]");
566
567 assertEquals(MultiTermQuery.CONSTANT_SCORE_REWRITE, ((TermRangeQuery)getQuery("[ a TO z]")).getRewriteMethod());
568
569 CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.SIMPLE, true));
570
571 qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
572 assertEquals(MultiTermQuery.SCORING_BOOLEAN_REWRITE,((TermRangeQuery)getQuery("[ a TO z]", qp)).getRewriteMethod());
573
574
575 assertQueryEquals("[ a TO * ]", null, "[a TO *]");
576 assertQueryEquals("[ * TO z ]", null, "[* TO z]");
577 assertQueryEquals("[ * TO * ]", null, "[* TO *]");
578
579
580 assertQueryEquals("{ a TO z ]", null, "{a TO z]");
581 assertQueryEquals("[ a TO z }", null, "[a TO z}");
582 assertQueryEquals("{ a TO * ]", null, "{a TO *]");
583 assertQueryEquals("[ * TO z }", null, "[* TO z}");
584
585 assertQueryEquals("[ a TO z ]", null, "[a TO z]");
586 assertQueryEquals("{ a TO z}", null, "{a TO z}");
587 assertQueryEquals("{ a TO z }", null, "{a TO z}");
588 assertQueryEquals("{ a TO z }^2.0", null, "{a TO z}^2.0");
589 assertQueryEquals("[ a TO z] OR bar", null, "[a TO z] bar");
590 assertQueryEquals("[ a TO z] AND bar", null, "+[a TO z] +bar");
591 assertQueryEquals("( bar blar { a TO z}) ", null, "bar blar {a TO z}");
592 assertQueryEquals("gack ( bar blar { a TO z}) ", null, "gack (bar blar {a TO z})");
593
594 assertQueryEquals("[* TO Z]",null,"[* TO z]");
595 assertQueryEquals("[A TO *]",null,"[a TO *]");
596 assertQueryEquals("[* TO *]",null,"[* TO *]");
597 }
598
599 public void testRangeWithPhrase() throws Exception {
600 assertQueryEquals("[\\* TO \"*\"]",null,"[\\* TO \\*]");
601 assertQueryEquals("[\"*\" TO *]",null,"[\\* TO *]");
602 }
603
604 private String escapeDateString(String s) {
605 if (s.indexOf(" ") > -1) {
606 return "\"" + s + "\"";
607 } else {
608 return s;
609 }
610 }
611
612
613 private String getDate(String s, DateTools.Resolution resolution) throws Exception {
614
615 DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault());
616 return getDate(df.parse(s), resolution);
617 }
618
619
620 private String getDate(Date d, DateTools.Resolution resolution) {
621 return DateTools.dateToString(d, resolution);
622 }
623
624 private String getLocalizedDate(int year, int month, int day) {
625
626 DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault());
627 Calendar calendar = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault());
628 calendar.clear();
629 calendar.set(year, month, day);
630 calendar.set(Calendar.HOUR_OF_DAY, 23);
631 calendar.set(Calendar.MINUTE, 59);
632 calendar.set(Calendar.SECOND, 59);
633 calendar.set(Calendar.MILLISECOND, 999);
634 return df.format(calendar.getTime());
635 }
636
637 public void testDateRange() throws Exception {
638 String startDate = getLocalizedDate(2002, 1, 1);
639 String endDate = getLocalizedDate(2002, 1, 4);
640
641 Calendar endDateExpected = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault());
642 endDateExpected.clear();
643 endDateExpected.set(2002, 1, 4, 23, 59, 59);
644 endDateExpected.set(Calendar.MILLISECOND, 999);
645 final String defaultField = "default";
646 final String monthField = "month";
647 final String hourField = "hour";
648 Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
649 CommonQueryParserConfiguration qp = getParserConfig(a);
650
651
652 setDateResolution(qp, monthField, DateTools.Resolution.MONTH);
653
654
655 qp.setDateResolution(DateTools.Resolution.MILLISECOND);
656
657
658 setDateResolution(qp, hourField, DateTools.Resolution.HOUR);
659
660
661
662 assertDateRangeQueryEquals(qp, defaultField, startDate, endDate,
663 endDateExpected.getTime(), DateTools.Resolution.MILLISECOND);
664
665
666 assertDateRangeQueryEquals(qp, monthField, startDate, endDate,
667 endDateExpected.getTime(), DateTools.Resolution.MONTH);
668
669 assertDateRangeQueryEquals(qp, hourField, startDate, endDate,
670 endDateExpected.getTime(), DateTools.Resolution.HOUR);
671 }
672
673 public void assertDateRangeQueryEquals(CommonQueryParserConfiguration cqpC, String field, String startDate, String endDate,
674 Date endDateInclusive, DateTools.Resolution resolution) throws Exception {
675 assertQueryEquals(cqpC, field, field + ":[" + escapeDateString(startDate) + " TO " + escapeDateString(endDate) + "]",
676 "[" + getDate(startDate, resolution) + " TO " + getDate(endDateInclusive, resolution) + "]");
677 assertQueryEquals(cqpC, field, field + ":{" + escapeDateString(startDate) + " TO " + escapeDateString(endDate) + "}",
678 "{" + getDate(startDate, resolution) + " TO " + getDate(endDate, resolution) + "}");
679 }
680
681 public void testEscaped() throws Exception {
682 Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708 assertQueryEquals("\\a", a, "a");
709
710 assertQueryEquals("a\\-b:c", a, "a-b:c");
711 assertQueryEquals("a\\+b:c", a, "a+b:c");
712 assertQueryEquals("a\\:b:c", a, "a:b:c");
713 assertQueryEquals("a\\\\b:c", a, "a\\b:c");
714
715 assertQueryEquals("a:b\\-c", a, "a:b-c");
716 assertQueryEquals("a:b\\+c", a, "a:b+c");
717 assertQueryEquals("a:b\\:c", a, "a:b:c");
718 assertQueryEquals("a:b\\\\c", a, "a:b\\c");
719
720 assertQueryEquals("a:b\\-c*", a, "a:b-c*");
721 assertQueryEquals("a:b\\+c*", a, "a:b+c*");
722 assertQueryEquals("a:b\\:c*", a, "a:b:c*");
723
724 assertQueryEquals("a:b\\\\c*", a, "a:b\\c*");
725
726 assertQueryEquals("a:b\\-c~", a, "a:b-c~2");
727 assertQueryEquals("a:b\\+c~", a, "a:b+c~2");
728 assertQueryEquals("a:b\\:c~", a, "a:b:c~2");
729 assertQueryEquals("a:b\\\\c~", a, "a:b\\c~2");
730
731 assertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]");
732 assertQueryEquals("[ a\\: TO a\\~ ]", null, "[a: TO a~]");
733 assertQueryEquals("[ a\\\\ TO a\\* ]", null, "[a\\ TO a*]");
734
735 assertQueryEquals("[\"c\\:\\\\temp\\\\\\~foo0.txt\" TO \"c\\:\\\\temp\\\\\\~foo9.txt\"]", a,
736 "[c:\\temp\\~foo0.txt TO c:\\temp\\~foo9.txt]");
737
738 assertQueryEquals("a\\\\\\+b", a, "a\\+b");
739
740 assertQueryEquals("a \\\"b c\\\" d", a, "a \"b c\" d");
741 assertQueryEquals("\"a \\\"b c\\\" d\"", a, "\"a \"b c\" d\"");
742 assertQueryEquals("\"a \\+b c d\"", a, "\"a +b c d\"");
743
744 assertQueryEquals("c\\:\\\\temp\\\\\\~foo.txt", a, "c:\\temp\\~foo.txt");
745
746 assertParseException("XY\\");
747
748
749 assertQueryEquals("a\\u0062c", a, "abc");
750 assertQueryEquals("XY\\u005a", a, "XYZ");
751 assertQueryEquals("XY\\u005A", a, "XYZ");
752 assertQueryEquals("\"a \\\\\\u0028\\u0062\\\" c\"", a, "\"a \\(b\" c\"");
753
754 assertParseException("XY\\u005G");
755 assertParseException("XY\\u005");
756
757
758 assertQueryEquals("(item:\\\\ item:ABCD\\\\)", a, "item:\\ item:ABCD\\");
759 assertParseException("(item:\\\\ item:ABCD\\\\))");
760 assertQueryEquals("\\*", a, "*");
761 assertQueryEquals("\\\\", a, "\\");
762
763 assertParseException("\\");
764
765
766 assertQueryEquals("(\"a\\\\\") or (\"b\")", a ,"a\\ or b");
767 }
768
769 public void testEscapedVsQuestionMarkAsWildcard() throws Exception {
770 Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
771 assertQueryEquals("a:b\\-?c", a, "a:b\\-?c");
772 assertQueryEquals("a:b\\+?c", a, "a:b\\+?c");
773 assertQueryEquals("a:b\\:?c", a, "a:b\\:?c");
774
775 assertQueryEquals("a:b\\\\?c", a, "a:b\\\\?c");
776 }
777
778 public void testQueryStringEscaping() throws Exception {
779 Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
780
781 assertEscapedQueryEquals("a-b:c", a, "a\\-b\\:c");
782 assertEscapedQueryEquals("a+b:c", a, "a\\+b\\:c");
783 assertEscapedQueryEquals("a:b:c", a, "a\\:b\\:c");
784 assertEscapedQueryEquals("a\\b:c", a, "a\\\\b\\:c");
785
786 assertEscapedQueryEquals("a:b-c", a, "a\\:b\\-c");
787 assertEscapedQueryEquals("a:b+c", a, "a\\:b\\+c");
788 assertEscapedQueryEquals("a:b:c", a, "a\\:b\\:c");
789 assertEscapedQueryEquals("a:b\\c", a, "a\\:b\\\\c");
790
791 assertEscapedQueryEquals("a:b-c*", a, "a\\:b\\-c\\*");
792 assertEscapedQueryEquals("a:b+c*", a, "a\\:b\\+c\\*");
793 assertEscapedQueryEquals("a:b:c*", a, "a\\:b\\:c\\*");
794
795 assertEscapedQueryEquals("a:b\\\\c*", a, "a\\:b\\\\\\\\c\\*");
796
797 assertEscapedQueryEquals("a:b-?c", a, "a\\:b\\-\\?c");
798 assertEscapedQueryEquals("a:b+?c", a, "a\\:b\\+\\?c");
799 assertEscapedQueryEquals("a:b:?c", a, "a\\:b\\:\\?c");
800
801 assertEscapedQueryEquals("a:b?c", a, "a\\:b\\?c");
802
803 assertEscapedQueryEquals("a:b-c~", a, "a\\:b\\-c\\~");
804 assertEscapedQueryEquals("a:b+c~", a, "a\\:b\\+c\\~");
805 assertEscapedQueryEquals("a:b:c~", a, "a\\:b\\:c\\~");
806 assertEscapedQueryEquals("a:b\\c~", a, "a\\:b\\\\c\\~");
807
808 assertEscapedQueryEquals("[ a - TO a+ ]", null, "\\[ a \\- TO a\\+ \\]");
809 assertEscapedQueryEquals("[ a : TO a~ ]", null, "\\[ a \\: TO a\\~ \\]");
810 assertEscapedQueryEquals("[ a\\ TO a* ]", null, "\\[ a\\\\ TO a\\* \\]");
811
812
813 assertEscapedQueryEquals("|| abc ||", a, "\\|\\| abc \\|\\|");
814 assertEscapedQueryEquals("&& abc &&", a, "\\&\\& abc \\&\\&");
815 }
816
817 public void testTabNewlineCarriageReturn()
818 throws Exception {
819 assertQueryEqualsDOA("+weltbank +worlbank", null,
820 "+weltbank +worlbank");
821
822 assertQueryEqualsDOA("+weltbank\n+worlbank", null,
823 "+weltbank +worlbank");
824 assertQueryEqualsDOA("weltbank \n+worlbank", null,
825 "+weltbank +worlbank");
826 assertQueryEqualsDOA("weltbank \n +worlbank", null,
827 "+weltbank +worlbank");
828
829 assertQueryEqualsDOA("+weltbank\r+worlbank", null,
830 "+weltbank +worlbank");
831 assertQueryEqualsDOA("weltbank \r+worlbank", null,
832 "+weltbank +worlbank");
833 assertQueryEqualsDOA("weltbank \r +worlbank", null,
834 "+weltbank +worlbank");
835
836 assertQueryEqualsDOA("+weltbank\r\n+worlbank", null,
837 "+weltbank +worlbank");
838 assertQueryEqualsDOA("weltbank \r\n+worlbank", null,
839 "+weltbank +worlbank");
840 assertQueryEqualsDOA("weltbank \r\n +worlbank", null,
841 "+weltbank +worlbank");
842 assertQueryEqualsDOA("weltbank \r \n +worlbank", null,
843 "+weltbank +worlbank");
844
845 assertQueryEqualsDOA("+weltbank\t+worlbank", null,
846 "+weltbank +worlbank");
847 assertQueryEqualsDOA("weltbank \t+worlbank", null,
848 "+weltbank +worlbank");
849 assertQueryEqualsDOA("weltbank \t +worlbank", null,
850 "+weltbank +worlbank");
851 }
852
853 public void testSimpleDAO()
854 throws Exception {
855 assertQueryEqualsDOA("term term term", null, "+term +term +term");
856 assertQueryEqualsDOA("term +term term", null, "+term +term +term");
857 assertQueryEqualsDOA("term term +term", null, "+term +term +term");
858 assertQueryEqualsDOA("term +term +term", null, "+term +term +term");
859 assertQueryEqualsDOA("-term term term", null, "-term +term +term");
860 }
861
862 public void testBoost()
863 throws Exception {
864 CharacterRunAutomaton stopWords = new CharacterRunAutomaton(Automata.makeString("on"));
865 Analyzer oneStopAnalyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopWords);
866 CommonQueryParserConfiguration qp = getParserConfig(oneStopAnalyzer);
867 Query q = getQuery("on^1.0",qp);
868 assertNotNull(q);
869 q = getQuery("\"hello\"^2.0",qp);
870 assertNotNull(q);
871 assertEquals(((BoostQuery) q).getBoost(), (float) 2.0, (float) 0.5);
872 q = getQuery("hello^2.0",qp);
873 assertNotNull(q);
874 assertEquals(((BoostQuery) q).getBoost(), (float) 2.0, (float) 0.5);
875 q = getQuery("\"on\"^1.0",qp);
876 assertNotNull(q);
877
878 Analyzer a2 = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
879 CommonQueryParserConfiguration qp2 = getParserConfig(a2);
880 q = getQuery("the^3", qp2);
881
882 assertNotNull(q);
883 assertEquals("", q.toString());
884 assertFalse(q instanceof BoostQuery);
885 }
886
887 public void assertParseException(String queryString) throws Exception {
888 try {
889 getQuery(queryString);
890 } catch (Exception expected) {
891 if(isQueryParserException(expected)){
892 return;
893 }
894 }
895 fail("ParseException expected, not thrown");
896 }
897
898 public void assertParseException(String queryString, Analyzer a) throws Exception {
899 try {
900 getQuery(queryString, a);
901 } catch (Exception expected) {
902 if(isQueryParserException(expected)){
903 return;
904 }
905 }
906 fail("ParseException expected, not thrown");
907 }
908
909 public void testException() throws Exception {
910 assertParseException("\"some phrase");
911 assertParseException("(foo bar");
912 assertParseException("foo bar))");
913 assertParseException("field:term:with:colon some more terms");
914 assertParseException("(sub query)^5.0^2.0 plus more");
915 assertParseException("secret AND illegal) AND access:confidential");
916 }
917
918 public void testBooleanQuery() throws Exception {
919 BooleanQuery.setMaxClauseCount(2);
920 Analyzer purWhitespaceAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
921 assertParseException("one two three", purWhitespaceAnalyzer);
922 }
923
924
925
926
927 public void testPrecedence() throws Exception {
928 CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
929 Query query1 = getQuery("A AND B OR C AND D", qp);
930 Query query2 = getQuery("+A +B +C +D", qp);
931 assertEquals(query1, query2);
932 }
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962 public abstract void testStarParsing() throws Exception;
963
964 public void testEscapedWildcard() throws Exception {
965 CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
966 WildcardQuery q = new WildcardQuery(new Term("field", "foo\\?ba?r"));
967 assertEquals(q, getQuery("foo\\?ba?r", qp));
968 }
969
970 public void testRegexps() throws Exception {
971 CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
972 RegexpQuery q = new RegexpQuery(new Term("field", "[a-z][123]"));
973 assertEquals(q, getQuery("/[a-z][123]/",qp));
974 qp.setLowercaseExpandedTerms(true);
975 assertEquals(q, getQuery("/[A-Z][123]/",qp));
976 assertEquals(new BoostQuery(q, 0.5f), getQuery("/[A-Z][123]/^0.5",qp));
977 qp.setMultiTermRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
978 q.setRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_REWRITE);
979 assertTrue(getQuery("/[A-Z][123]/^0.5",qp) instanceof BoostQuery);
980 assertTrue(((BoostQuery) getQuery("/[A-Z][123]/^0.5",qp)).getQuery() instanceof RegexpQuery);
981 assertEquals(MultiTermQuery.SCORING_BOOLEAN_REWRITE, ((RegexpQuery) ((BoostQuery) getQuery("/[A-Z][123]/^0.5",qp)).getQuery()).getRewriteMethod());
982 assertEquals(new BoostQuery(q, 0.5f), getQuery("/[A-Z][123]/^0.5",qp));
983 qp.setMultiTermRewriteMethod(MultiTermQuery.CONSTANT_SCORE_REWRITE);
984
985 Query escaped = new RegexpQuery(new Term("field", "[a-z]\\/[123]"));
986 assertEquals(escaped, getQuery("/[a-z]\\/[123]/",qp));
987 Query escaped2 = new RegexpQuery(new Term("field", "[a-z]\\*[123]"));
988 assertEquals(escaped2, getQuery("/[a-z]\\*[123]/",qp));
989
990 BooleanQuery.Builder complex = new BooleanQuery.Builder();
991 complex.add(new RegexpQuery(new Term("field", "[a-z]\\/[123]")), Occur.MUST);
992 complex.add(new TermQuery(new Term("path", "/etc/init.d/")), Occur.MUST);
993 complex.add(new TermQuery(new Term("field", "/etc/init[.]d/lucene/")), Occur.SHOULD);
994 assertEquals(complex.build(), getQuery("/[a-z]\\/[123]/ AND path:\"/etc/init.d/\" OR \"/etc\\/init\\[.\\]d/lucene/\" ",qp));
995
996 Query re = new RegexpQuery(new Term("field", "http.*"));
997 assertEquals(re, getQuery("field:/http.*/",qp));
998 assertEquals(re, getQuery("/http.*/",qp));
999
1000 re = new RegexpQuery(new Term("field", "http~0.5"));
1001 assertEquals(re, getQuery("field:/http~0.5/",qp));
1002 assertEquals(re, getQuery("/http~0.5/",qp));
1003
1004 re = new RegexpQuery(new Term("field", "boo"));
1005 assertEquals(re, getQuery("field:/boo/",qp));
1006 assertEquals(re, getQuery("/boo/",qp));
1007
1008 assertEquals(new TermQuery(new Term("field", "/boo/")), getQuery("\"/boo/\"",qp));
1009 assertEquals(new TermQuery(new Term("field", "/boo/")), getQuery("\\/boo\\/",qp));
1010
1011 BooleanQuery.Builder two = new BooleanQuery.Builder();
1012 two.add(new RegexpQuery(new Term("field", "foo")), Occur.SHOULD);
1013 two.add(new RegexpQuery(new Term("field", "bar")), Occur.SHOULD);
1014 assertEquals(two.build(), getQuery("field:/foo/ field:/bar/",qp));
1015 assertEquals(two.build(), getQuery("/foo/ /bar/",qp));
1016 }
1017
1018 public void testStopwords() throws Exception {
1019 CharacterRunAutomaton stopSet = new CharacterRunAutomaton(new RegExp("the|foo").toAutomaton());
1020 CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, stopSet));
1021 Query result = getQuery("field:the OR field:foo",qp);
1022 assertNotNull("result is null and it shouldn't be", result);
1023 assertTrue("result is not a BooleanQuery", result instanceof BooleanQuery || result instanceof MatchNoDocsQuery);
1024 if (result instanceof BooleanQuery) {
1025 assertEquals(0, ((BooleanQuery) result).clauses().size());
1026 }
1027 result = getQuery("field:woo OR field:the",qp);
1028 assertNotNull("result is null and it shouldn't be", result);
1029 assertTrue("result is not a TermQuery", result instanceof TermQuery);
1030 result = getQuery("(fieldX:xxxxx OR fieldy:xxxxxxxx)^2 AND (fieldx:the OR fieldy:foo)",qp);
1031 assertNotNull("result is null and it shouldn't be", result);
1032 assertTrue("result is not a BoostQuery", result instanceof BoostQuery);
1033 result = ((BoostQuery) result).getQuery();
1034 assertTrue("result is not a BooleanQuery", result instanceof BooleanQuery);
1035 if (VERBOSE) System.out.println("Result: " + result);
1036 assertTrue(((BooleanQuery) result).clauses().size() + " does not equal: " + 2, ((BooleanQuery) result).clauses().size() == 2);
1037 }
1038
1039 public void testPositionIncrement() throws Exception {
1040 CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET));
1041 qp.setEnablePositionIncrements(true);
1042 String qtxt = "\"the words in poisitions pos02578 are stopped in this phrasequery\"";
1043
1044 int expectedPositions[] = {1,3,4,6,9};
1045 PhraseQuery pq = (PhraseQuery) getQuery(qtxt,qp);
1046
1047
1048 Term t[] = pq.getTerms();
1049 int pos[] = pq.getPositions();
1050 for (int i = 0; i < t.length; i++) {
1051
1052 assertEquals("term "+i+" = "+t[i]+" has wrong term-position!",expectedPositions[i],pos[i]);
1053 }
1054 }
1055
1056 public void testMatchAllDocs() throws Exception {
1057 CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
1058 assertEquals(new MatchAllDocsQuery(), getQuery("*:*",qp));
1059 assertEquals(new MatchAllDocsQuery(), getQuery("(*:*)",qp));
1060 BooleanQuery bq = (BooleanQuery)getQuery("+*:* -*:*",qp);
1061 assertEquals(2, bq.clauses().size());
1062 for (BooleanClause clause : bq) {
1063 assertTrue(clause.getQuery() instanceof MatchAllDocsQuery);
1064 }
1065 }
1066
1067 @SuppressWarnings("unused")
1068 private void assertHits(int expected, String query, IndexSearcher is) throws Exception {
1069 String oldDefaultField = getDefaultField();
1070 setDefaultField("date");
1071 CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false));
1072 qp.setLocale(Locale.ENGLISH);
1073 Query q = getQuery(query,qp);
1074 ScoreDoc[] hits = is.search(q, 1000).scoreDocs;
1075 assertEquals(expected, hits.length);
1076 setDefaultField( oldDefaultField );
1077 }
1078
1079 @Override
1080 public void tearDown() throws Exception {
1081 BooleanQuery.setMaxClauseCount(originalMaxClauses);
1082 super.tearDown();
1083 }
1084
1085
1086
1087
1088 public void testPositionIncrements() throws Exception {
1089 Directory dir = newDirectory();
1090 Analyzer a = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
1091 IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(a));
1092 Document doc = new Document();
1093 doc.add(newTextField("field", "the wizard of ozzy", Field.Store.NO));
1094 w.addDocument(doc);
1095 IndexReader r = DirectoryReader.open(w, true);
1096 w.close();
1097 IndexSearcher s = newSearcher(r);
1098
1099 Query q = getQuery("\"wizard of ozzy\"",a);
1100 assertEquals(1, s.search(q, 1).totalHits);
1101 r.close();
1102 dir.close();
1103 }
1104
1105
1106
1107
1108 protected static class MockSynonymFilter extends TokenFilter {
1109 CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
1110 PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
1111 boolean addSynonym = false;
1112
1113 public MockSynonymFilter(TokenStream input) {
1114 super(input);
1115 }
1116
1117 @Override
1118 public final boolean incrementToken() throws IOException {
1119 if (addSynonym) {
1120 clearAttributes();
1121 termAtt.setEmpty().append("dog");
1122 posIncAtt.setPositionIncrement(0);
1123 addSynonym = false;
1124 return true;
1125 }
1126
1127 if (input.incrementToken()) {
1128 addSynonym = termAtt.toString().equals("dogs");
1129 return true;
1130 } else {
1131 return false;
1132 }
1133 }
1134 }
1135
1136
1137 protected class Analyzer1 extends Analyzer {
1138 public Analyzer1(){
1139 super();
1140 }
1141 @Override
1142 public TokenStreamComponents createComponents(String fieldName) {
1143 Tokenizer tokenizer = new MockTokenizer( MockTokenizer.WHITESPACE, true);
1144 return new TokenStreamComponents(tokenizer, new MockSynonymFilter(tokenizer));
1145 }
1146 }
1147
1148
1149 protected class Analyzer2 extends Analyzer {
1150 public Analyzer2(){
1151 super();
1152 }
1153 @Override
1154 public TokenStreamComponents createComponents(String fieldName) {
1155 return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
1156 }
1157 }
1158
1159 public abstract void testNewFieldQuery() throws Exception;
1160
1161
1162
1163
1164 private class MockCollationFilter extends TokenFilter {
1165 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
1166
1167 protected MockCollationFilter(TokenStream input) {
1168 super(input);
1169 }
1170
1171 @Override
1172 public boolean incrementToken() throws IOException {
1173 if (input.incrementToken()) {
1174 String term = termAtt.toString();
1175 termAtt.setEmpty().append("collated").append(term);
1176 return true;
1177 } else {
1178 return false;
1179 }
1180 }
1181
1182 }
1183 private class MockCollationAnalyzer extends Analyzer {
1184 @Override
1185 public TokenStreamComponents createComponents(String fieldName) {
1186 Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
1187 return new TokenStreamComponents(tokenizer, new MockCollationFilter(tokenizer));
1188 }
1189 }
1190
1191 public void testCollatedRange() throws Exception {
1192 CommonQueryParserConfiguration qp = getParserConfig(new MockCollationAnalyzer());
1193 setAnalyzeRangeTerms(qp, true);
1194 Query expected = TermRangeQuery.newStringRange(getDefaultField(), "collatedabc", "collateddef", true, true);
1195 Query actual = getQuery("[abc TO def]", qp);
1196 assertEquals(expected, actual);
1197 }
1198
1199 public void testDistanceAsEditsParsing() throws Exception {
1200 FuzzyQuery q = (FuzzyQuery) getQuery("foobar~2",new MockAnalyzer(random()));
1201 assertEquals(2, q.getMaxEdits());
1202 }
1203
1204 public void testPhraseQueryToString() throws Exception {
1205 Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET);
1206 CommonQueryParserConfiguration qp = getParserConfig(analyzer);
1207 qp.setEnablePositionIncrements(true);
1208 PhraseQuery q = (PhraseQuery)getQuery("\"this hi this is a test is\"", qp);
1209 assertEquals("field:\"? hi ? ? ? test\"", q.toString());
1210 }
1211
1212 public void testParseWildcardAndPhraseQueries() throws Exception {
1213 String field = "content";
1214 String oldDefaultField = getDefaultField();
1215 setDefaultField(field);
1216 CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random()));
1217 qp.setAllowLeadingWildcard(true);
1218
1219 String prefixQueries[][] = {
1220 {"a*", "ab*", "abc*",},
1221 {"h*", "hi*", "hij*", "\\\\7*"},
1222 {"o*", "op*", "opq*", "\\\\\\\\*"},
1223 };
1224
1225 String wildcardQueries[][] = {
1226 {"*a*", "*ab*", "*abc**", "ab*e*", "*g?", "*f?1", "abc**"},
1227 {"*h*", "*hi*", "*hij**", "hi*k*", "*n?", "*m?1", "hij**"},
1228 {"*o*", "*op*", "*opq**", "op*q*", "*u?", "*t?1", "opq**"},
1229 };
1230
1231
1232 for (int i = 0; i < prefixQueries.length; i++) {
1233 for (int j = 0; j < prefixQueries[i].length; j++) {
1234 String queryString = prefixQueries[i][j];
1235 Query q = getQuery(queryString,qp);
1236 assertEquals(PrefixQuery.class, q.getClass());
1237 }
1238 }
1239
1240
1241 for (int i = 0; i < wildcardQueries.length; i++) {
1242 for (int j = 0; j < wildcardQueries[i].length; j++) {
1243 String qtxt = wildcardQueries[i][j];
1244 Query q = getQuery(qtxt,qp);
1245 assertEquals(WildcardQuery.class, q.getClass());
1246 }
1247 }
1248 setDefaultField(oldDefaultField);
1249 }
1250
1251 public void testPhraseQueryPositionIncrements() throws Exception {
1252 CharacterRunAutomaton stopStopList =
1253 new CharacterRunAutomaton(new RegExp("[sS][tT][oO][pP]").toAutomaton());
1254
1255 CommonQueryParserConfiguration qp = getParserConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList));
1256
1257 qp = getParserConfig(
1258 new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false, stopStopList));
1259 qp.setEnablePositionIncrements(true);
1260
1261 PhraseQuery.Builder phraseQuery = new PhraseQuery.Builder();
1262 phraseQuery.add(new Term("field", "1"));
1263 phraseQuery.add(new Term("field", "2"), 2);
1264 assertEquals(phraseQuery.build(), getQuery("\"1 stop 2\"",qp));
1265 }
1266
1267 public void testMatchAllQueryParsing() throws Exception {
1268
1269 String oldDefaultField = getDefaultField();
1270 setDefaultField("key");
1271 CommonQueryParserConfiguration qp = getParserConfig( new MockAnalyzer(random()));
1272 assertEquals(new MatchAllDocsQuery(), getQuery(new MatchAllDocsQuery().toString(),qp));
1273
1274
1275 Query query = new MatchAllDocsQuery();
1276 query = new BoostQuery(query, 2.3f);
1277 assertEquals(query, getQuery(query.toString(),qp));
1278 setDefaultField(oldDefaultField);
1279 }
1280
1281 public void testNestedAndClausesFoo() throws Exception {
1282 String query = "(field1:[1 TO *] AND field1:[* TO 2]) AND field2:(z)";
1283 BooleanQuery.Builder q = new BooleanQuery.Builder();
1284 BooleanQuery.Builder bq = new BooleanQuery.Builder();
1285 bq.add(TermRangeQuery.newStringRange("field1", "1", null, true, true), BooleanClause.Occur.MUST);
1286 bq.add(TermRangeQuery.newStringRange("field1", null, "2", true, true), BooleanClause.Occur.MUST);
1287 q.add(bq.build(), BooleanClause.Occur.MUST);
1288 q.add(new TermQuery(new Term("field2", "z")), BooleanClause.Occur.MUST);
1289 assertEquals(q.build(), getQuery(query, new MockAnalyzer(random())));
1290 }
1291 }